# -*- coding: utf-8 -*-
# @Time    : 2018/9/7 11:31
import threading
import os
path = os.path.abspath(os.path.dirname(os.getcwd()))
import sys
sys.path.append(path)
import requests
import pymongo,re
import config
from hashlib import md5
from time import sleep
from collections import OrderedDict
from lxml.html import etree
from multiprocessing import Process, Queue, Pool



headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.12 Safari/537.36',
           'cookie': 'DUP=Q=Ga0YFBMlCNGhWMlllnXhdQ2&T=339134409&A=2&IG=794C85C09878432F8CDBE420E943BA96; SRCHD=AF=NOFORM; SRCHUID=V=2&GUID=AEEE921BC034492E9A58E2075DBAA285&dmnchg=1; _EDGE_V=1; MUID=3D1EEA84DEBC6C601E39E6D7DF926D68; MUIDB=3D1EEA84DEBC6C601E39E6D7DF926D68; ULC=P=1638A|4:4&H=1638A|4:4&T=1638A|4:4:1; _EDGE_S=mkt=zh-cn&SID=19906E7F5024635726DD6205510A6251; ipv6=hit=1538283079437&t=4; SRCHHPGUSR=CW=1309&CH=326&DPR=1&UTC=480&WTS=63673876278; _FP=hta=on; ENSEARCH=BENVER=0; SRCHUSR=DOB=20180822&T=1538282898000; _SS=SID=19906E7F5024635726DD6205510A6251&bIm=77:146&HV=1538282898',

}
db_million = config.db_million
url_0 = db_million['url_0']
url_1 = db_million['url_1']
url_2 = db_million['url_2']
url_3 = db_million['url_3']
url_4 = db_million['url_4']
url_5 = db_million['url_5']
url_6 = db_million['url_6']
url_7 = db_million['url_7']
url_8 = db_million['url_8']
url_9 = db_million['url_9']
url_a = db_million['url_a']
url_b = db_million['url_b']
url_c = db_million['url_c']
url_d = db_million['url_d']
url_e = db_million['url_e']
url_f = db_million['url_f']
url_s = {"0":url_0, "1":url_1, "2":url_2, "3":url_3, "4":url_4, "5":url_5, "6":url_6, "7":url_7, "8":url_8,
         "9":url_9, "a":url_a, "b":url_b, "c":url_c, "d":url_d, "e":url_e, "f":url_f}
url_biying = config.li_db_biying
new_url_biying = config.new_li_db_biying

class BiYing(object):
    def __init__(self) -> None:
        pass
        # self.pro = self.get_pro()

    # def get_pro(self):
    #     db_one = pymongo.MongoClient(config.MONGODB_URI)['proxy']
    #     li_db_one = db_one['proxies_ip']
    #     xx_pro = []
    #     li_db_two = li_db_one.find()
    #     for i in li_db_two:
    #         ip = i['ip']
    #         xx_pro.append({"http":ip})
    #     return xx_pro

    def find_one(self,item):
        md5 = item['url_md5']
        if url_s["%s" % (md5[0])].find({"md5":md5}).count():
            return
        else:
            #插入数据
            print(md5)
            try:
                new_url_biying.insert(item)
            except Exception as e:
                if str(e).find("E11000") > -1:
                    return
                print("插入失败：%s" % (e))


    def repl(self, text):
        try:
            new_text = re.sub(r"[\n\t\r\u3000\xa0\u2002]","",text).strip()
            return new_text
        except:
            return text

    def delete_mongo(self):

        while True:
            try:
                # 关键词
                item = url_biying.find_and_modify({'state':0}, {'$set': {"state":1}})
                if not item:
                    return
                self.find_one(item)
            except Exception as e:
                print(e)
                # biying_log.info("error delete_mongo: %s"%(e))

    def md5_generator(self,url):
        return md5(url.encode()).hexdigest()

    def proce(self):
        trader = []
        for i in range(24):
            pr = Process(target=self.delete_carw)
            sleep(0.5)
            pr.start()
            trader.append(pr)
        for i in trader:
            i.join()
        print('proce this is pid: %s' % os.getpid())

    def delete_carw(self):
        threads = []
        for i in range(5):
            t = threading.Thread(target=self.delete_mongo, name='LoopThread_%s'% i)
            threads.append(t)
        for t in threads:
            sleep(0.5)
            t.start()
        for t in threads:
            # 多线程多join的情况下，依次执行各线程的join方法, 这样可以确保主线程最后退出， 且各个线程间没有阻塞
            t.join()
        print('delete this is pid: %s' % os.getpid())
        return



if __name__ == '__main__':
    BiYing().proce()
    # print(BiYing().md5_generator("http://www.sda.gov.cn/directory/web/WS01/images/MjAxN8Tqtdo1usW5q7jmuL28i5kb2N4.docx"))


